import os
import re
import json
import string
import csv
import shutil


def check_contain_chinese(check_str):
    for ch in check_str:
        if u'\u4e00' > ch or ch>u'\u9fff':
            return False
    return True
def substring_before(s, delim):
    if not s:
        return ''
    if delim in s:
        return s.partition(delim)[0]
    else:
        return s
def substring_after(s, delim):
    if not s:
        return ''
    if delim in s:
        return s.partition(delim)[2]
    else:
        return ''
def judge_pure_english(keyword):
    return all(c.isupper() or c.islower() for c in keyword)
def check(str):
    my_re = re.compile(r'[A-Za-z]',re.S)
    res = re.findall(my_re,str)
    if len(res):
        return True
    else:
        return False
def filelinecount(filename):
    count = -1  
    for count,line in enumerate(open(filename,'r',encoding="utf-8")):
        pass  
    count += 1
    return count
def TextFileSearch(txtFile,string):
    have=False
    #下面的encoding可指定编码，如我要查找的文件类型是html文件，所以编码设为utf-8
    f = open(txtFile, 'r', encoding='utf-8')
    for line in f.readlines():
        tline=line.strip().rstrip()
        #print(string)
        #print(tline)
        if string==tline:
            #print("存在")
            have=True
            break
    f.close()
    return have
_MAP = {
    "!": "PUNCT",
    '"': "PUNCT",
    "#": "X",
    "'": "PUNCT",
    ",": "PUNCT",
    "-": "PUNCT",
    ".": "PUNCT",
    "...": "PUNCT",
    "?": "PUNCT",
    
    "INTJ": "e",  # HKCanCor: Interjection  感叹词
    "PRO": "r",  # HKCanCor: Pronoun
    "VERB": "v",  # HKCanCor: Verb
    "NOUN": "n",  # Common Noun
    "ADJ": "a",  # HKCanCor: Adjective
    "PART": "h", # HKCanCor: Prefix (aa3 阿 followed by a person name) 前缀
    "NUM": "m",  # HKCanCor: Numeral 数词
    "CL": "q",  #HKCanCor: Classifier 量词 个体单位:个、位、件、本、间、把、条、根、棵、张、匹、块、片、 只、架、枝、艘、辆、幅、座、句、段、篇、章、首 集体单位:双、对、副、堆、批、群、帮,班
    "ASP": "u",  # HKCanCor: Auxiliary (e.g., ge3 嘅 after an attributive adj) 助动词
    "PREP": "p",  # HKCanCor: Preposition 介词
    "ADV": "ad",  # HKCanCor: Adjective as Adverbial
    "SFP": "y",  # HKCanCor:Modal Particle 语气助词
    "CONJ": "c",  # HKCanCor: Conjunction  连词 
    "DET": "n1",  # HKCanCor: only used for ne1 呢 determiner in the upstream data
    "IDIOM": "i", # HKCanCor: Idiom  成语
    "X": "x",  # HKCanCor: Unclassified Item
    "ONO": "o",  # HKCanCor:Onomatopoeia 拟声词
    
    " ": " ",  # HKCanCor:error
    "CCONJ": "c",  # HKCanCor: Conjunction
    "PROPN": "XS",
    "PRON": "r",  # HKCanCor: Pronoun   
    "G": "X",  # HKCanCor: Morpheme
    "G1": "V",  # The first A in the "A-not-AB" pattern, where AB is a verb.
    "I": "X",  # HKCanCor: Idiom
    "IG": "X",
    "K": "X",  # HKCanCor: Suffix (sing3 性 for nouns; dei6 地 for adverbs)
    "L": "X",  # Fixed Expression
    "L1": "X",
    "LG": "X",   
    "MG": "X",
    "O": "X",  # HKCanCor: Onomatopoeia 拟声词
    "VU": "AUX",
    "VUG": "AUX",
    "W": "PUNCT",  # HKCanCor: Punctuation    
    "XJ": "X",
    "XX": "X",
}

path1= r'.\V2\test'  #定义原始电影对白数据文件夹路径
path2= r'.\V2\Genre' #定义导演信息文件夹路径
path4= r'.\V2\out'   #定义输出信息文件夹的路径
path6= r'.\V2\tmp'   #定义临时文件夹的路径
path7= r'.\V2\chat'  #定义按电影名整理好的电影对白数据的文件夹路径
path8= r'.\V2\cha'   #定义电影名对应电影的文件头数据的文件夹路径
path9= r'.\V2\tmp_sorted'  #定义临时数据排序后文件的文件夹的路径
file_names = os.listdir(path1) #创建一个所有文件名的列表
if not os.path.exists(path4):
    os.mkdir(path4)
if not os.path.exists(path6):
    os.mkdir(path6)
if not os.path.exists(path7):
    os.mkdir(path7)
if not os.path.exists(path8):
    os.mkdir(path8)
if not os.path.exists(path9):
    os.mkdir(path9)
zwbd=r"！？。＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏."
punc =list(zwbd)
wnoin=set()
#wf4=open(path4+'\\'+"url_list.txt","w",encoding="utf-8")
#wf4.close()
wf4=open(path4+'\\'+"Warning.txt","w",encoding="utf-8")
wf4.close()
wf4=open(path4+'\\'+"Error.txt","w",encoding="utf-8")
wf4.close()
wf4=open(path4+'\\'+"poslist.txt","w",encoding="utf-8")
wf4.close()
url_list=[]
sent_list=[]
fgenredict={}
wordPOSDict={}
wordJyutpingDict={}
twowordJyutpingDict={}
poscountlist=[]
poslist=[]
genrefile=path2+'\\'+"genre_file.csv"

'''
@UTF8
@Begin
@Languages:	yue , eng
@Participants:	XXA A Adult , XXB B Adult
@ID:	yue , eng|HKCC|XXA|26;|female|||Adult||origin:HK|
@ID:	yue , eng|HKCC|XXB|24;|female|||Adult||origin:HK|
@Film Name:	 
'''
#filmname=''

with open(path2+'\\'+"genre_file.csv", encoding="gbk") as f:#根据导演数据文件生成导演字典
    reader = csv.reader(f)
    next(reader)
    
    for lineitems in reader:
        year=lineitems[0]
        filmname=lineitems[1]
        #filmname=substring_after(filmname, '/')
        filmname=substring_before(filmname, '/')
        if '(' in filmname:
            #print("有括号")
            fname1=substring_before(filmname, '(')
            fname2=substring_after(filmname, '(')
            fname2=substring_before(fname2, ')')
            fname1=fname1.strip().rstrip()
            fname2=fname2.strip().rstrip()
            filmname=fname1+"("+fname2+")"
        filmname=filmname.strip().rstrip()
        filmname=filmname.replace(' ','_')
        filmname=filmname.strip().rstrip()+" ("+year+")"
        genre=lineitems[2]
        #print(filmname)
        fgenredict.update({filmname:genre})
    #print(fgenredict)  

file_del= os.listdir(path6) #创建一个即将删除的所有文件名的列表
for name in file_del:
    if os.path.isfile(path6+'\\'+name):
        os.unlink(path6+'\\'+name)
file_del= os.listdir(path8) #创建一个即将删除的所有文件名的列表
for name in file_del:
    if os.path.isfile(path8+'\\'+name):
        os.unlink(path8+'\\'+name)

for name in file_names:
    print("正在处理"+name+"文件,由于数据量大，可能需要较长时间，请耐心等待......")
    fnamelist=name.split('.')
    #wf_chat=open(path6+'\\'+fnamelist[0]+".chat","w",encoding="utf-8")
    #wf_chat.close()
    with open(path1+'\\'+name, encoding="utf-8") as f:
        reader = csv.reader(f)
        next(reader)
        pre=0
        #pre=171
        lineNo=1
        for lineitems in reader:
            #print(lineitems)
            s = re.sub(r"[^0-9]", "", lineitems[0])
            Actor=lineitems[1]
            tmpactor=Actor
            Actor=substring_after(Actor, '/')
            Actor=substring_before(Actor, '[')
            Actor=Actor.strip().rstrip()
            Actor=substring_after(Actor, '(')
            Actor=substring_before(Actor, ')')
            Actor=Actor.strip().rstrip()#获取演员信息
            #@ 00:28:23 -
            time=lineitems[2]
            time=substring_after(time, '@')
            time=substring_before(time, '-')
            time=time.replace(' ','')
            hour=substring_before(time, ':')
            time=substring_after(time, ':')
            minute=substring_before(time, ':')
            second=substring_after(time, ':')
            sound_time=3600*int(hour)+60*int(minute)+int(second)#获取当前需要处理的电影对白的时间点信息
            Film=lineitems[2]
            #print(Film)
            tmpfilm=substring_before(Film, '@')
            zwfilm=substring_after(tmpfilm, '/')#获取中文电影名
            tmpfilm=tmpfilm.strip().rstrip()
            Film=substring_before(Film, '/')
            #print(Film)
            if '(' in Film:
                fname1=substring_before(Film, '(')
                fname2=substring_after(Film, '(')
                fname2=substring_before(fname2, ')')
                fname1=fname1.strip().rstrip()
                fname2=fname2.strip().rstrip()
                Film=fname1+"("+fname2+")"
            Film=Film.strip().rstrip()
            Film=Film.replace(' ','_')
            tFilm=substring_after(Film, '(')
            year=''
            
            if '） (' in zwfilm:
                year=substring_after(zwfilm,  '） (' )
                #print("year:"+year)
            else:
                year=substring_after(tmpfilm, '(')
                #print("year:"+year)
            year=substring_before(year, ')')
            year=year.strip().rstrip()
            
            
            Film=Film.strip().rstrip()+" ("+year+")"#英文电影名加发行年份
            
            if not os.path.exists(path8+'\\'+Film+".cha"):
                wf_chat=open(path8+'\\'+Film+".cha","w",encoding="utf-8")
                wf_chat.close()
            if not TextFileSearch(path8+'\\'+Film+".cha","@Film Name:"+tmpfilm):
                #生成电影名对应电影的文件头数据文件
                with open(path8+'\\'+Film+".cha","a+",encoding="utf-8") as tf:
                    
                    tf.write("@UTF8"+'\n')
                    tf.write("@Begin"+'\n')
                    tf.write("@The Corpus of Mid-20th Century Hong Kong Cantonese"+'\n')
                    tf.write("@Data sourc:The Education University of Hong Kong"+'\n')
                    tf.write("@Developed and converted by Huang Chunxiao(Yunnan University)"+'\n')
                    tf.write("@Languages:	yue , eng"+'\n')
                    tf.write("@Film Name:"+tmpfilm+'\n')
                    tf.write("@Genre:"+fgenredict[Film]+'\n')
                    tf.write("@Participants:"+'\n')
                    
            
            tmpactor=tmpactor.strip().rstrip()
            if not TextFileSearch(path8+'\\'+Film+".cha","@ID:	yue , eng|HKCC|"+tmpactor+"|origin:HK|"):
                #如果演员名字在这部电影里没出现过，就把该演员数据添加进文件
                with open(path8+'\\'+Film+".cha","a+",encoding="utf-8") as tf:
                    tf.write("@ID:	yue , eng|HKCC|"+tmpactor+"|origin:HK|"+'\n')
            
            sentence=lineitems[3]
            if sentence not in sent_list:
                sent_list.append(sentence)
            tmp=eval(s)
            if tmp==pre+1:
                
                j=4
                wordlist=[]
                morlist=[]
                tmpsent=''
                canword=lineitems[j]
                if not canword:
                    canword='口'
                canword=canword.strip().rstrip()
                tmpsent=tmpsent+canword
                wordlist.append(canword)
                Jyutping=lineitems[j+1]
                Jyutping=Jyutping.replace(' ','')
                pos=lineitems[j+2]
                pos=pos.strip().rstrip()
                
                if judge_pure_english(canword):
                    tcanword=canword
                else:
                    tcanword=canword+Jyutping
                
                poscountlist=wordPOSDict.get(canword,[])
                Jyutpingcountlist=wordJyutpingDict.get(canword,[])
                if pos and not poscountlist:
                    
                    poscountlist=[]
                    p_c_item={}
                    p_c_item.update({'POS':pos})
                    p_c_item.update({'count':1})
                    p_c_item.update({'第一次出现的句子':sentence})
                    poscountlist.append(p_c_item)
                    
                    wordPOSDict.update({canword:poscountlist})
                elif pos and poscountlist:
                    pcLlen=len(poscountlist)
                    
                    k=-1
                    tmpcount=0
                    for i in range(pcLlen):
                        tmpPOS=poscountlist[i].get('POS','')
                        
                        if tmpPOS==pos:#这个canword的pos词性已经出现过，count要加1
                            tmpcount=poscountlist[i].get('count',0)+1
                            poscountlist[i].update({'count':tmpcount})
                            
                            k=i
                            break
                    
                    if k==-1:
                        
                        p_c_item={}
                        p_c_item.update({'POS':pos})
                        p_c_item.update({'count':1})
                        p_c_item.update({'第一次出现的句子':sentence})
                        poscountlist.append(p_c_item)
                    
                    wordPOSDict.update({canword:poscountlist})

                #########
                if Jyutping and not Jyutpingcountlist:
                    
                    Jyutpingcountlist=[]
                    p_c_item={}
                    p_c_item.update({'Jyutping':Jyutping})
                    p_c_item.update({'count':1})
                    p_c_item.update({'第一次出现的句子':sentence})
                    Jyutpingcountlist.append(p_c_item)
                    
                    wordJyutpingDict.update({canword:Jyutpingcountlist})
                    if len(canword)>1:
                        twowordJyutpingDict.update({canword:Jyutpingcountlist})
                elif Jyutping and Jyutpingcountlist:
                    pcLlen=len(Jyutpingcountlist)
                    
                    k=-1
                    tmpcount=0
                    for i in range(pcLlen):
                        tmpJyutping=Jyutpingcountlist[i].get('Jyutping','')
                        
                        if tmpJyutping==Jyutping:#这个canword的pos词性已经出现过，count要加1
                            tmpcount=Jyutpingcountlist[i].get('count',0)+1
                            Jyutpingcountlist[i].update({'count':tmpcount})
                            
                            k=i
                            break
                    
                    if k==-1:
                        
                        p_c_item={}
                        p_c_item.update({'Jyutping':Jyutping})
                        p_c_item.update({'count':1})
                        p_c_item.update({'第一次出现的句子':sentence})
                        Jyutpingcountlist.append(p_c_item)
                    
                    wordJyutpingDict.update({canword:Jyutpingcountlist})
                    if len(canword)>1:
                        twowordJyutpingDict.update({canword:Jyutpingcountlist})
                
                if not pos:
                        pos=' '
                if canword not in punc:
                    mor=_MAP[pos]+'|'+Jyutping
                else:
                    mor=canword
                if judge_pure_english(canword):
                    
                        mor=_MAP[pos]+'|'+canword.lower()+'0'
                morlist.append(mor)
                mic="*"+Actor+":	"+canword
                mors="%mor:	"+mor
                j=j+3
                
                while j<len(lineitems):
                    if not lineitems[j] and j+1<len(lineitems) and not lineitems[j+1]:
                        break
                    canword=lineitems[j]
                    if not canword:
                        canword='口'
                    canword=canword.strip().rstrip()
                    tmpsent=tmpsent+canword
                    wordlist.append(canword)
                    Jyutping=lineitems[j+1]
                    Jyutping=Jyutping.replace(' ','')
                    
                    pos=lineitems[j+2]
                    pos=pos.strip().rstrip()
                    #if not pos:
                    #    pos=' '
                    if judge_pure_english(canword):
                        tcanword=canword
                    else:
                        tcanword=canword+Jyutping
                    
                    poscountlist=wordPOSDict.get(canword,[])
                    Jyutpingcountlist=wordJyutpingDict.get(canword,[])
                    if pos and not poscountlist:
                        poscountlist=[]
                        p_c_item={}
                        p_c_item.update({'POS':pos})
                        p_c_item.update({'count':1})
                        p_c_item.update({'第一次出现的句子':sentence})
                        poscountlist.append(p_c_item)
                        
                        wordPOSDict.update({canword:poscountlist})
                        if len(canword)>1:
                            twowordJyutpingDict.update({canword:Jyutpingcountlist})
                    elif pos and poscountlist:
                        
                        pcLlen=len(poscountlist)
                        
                        k=-1
                        tmpcount=0
                        for i in range(pcLlen):
                            tmpPOS=poscountlist[i].get('POS','')
                            
                            if tmpPOS==pos:#这个canword的pos词性已经出现过，count要加1
                                tmpcount=poscountlist[i].get('count',0)+1
                                poscountlist[i].update({'count':tmpcount})
                                
                                k=i
                                break
                       
                        if k==-1:#这个canword的pos词性是第一次出现
                            p_c_item={}
                            
                            p_c_item.update({'POS':pos})
                            p_c_item.update({'count':1})
                            p_c_item.update({'第一次出现的句子':sentence})
                            poscountlist.append(p_c_item)
                        
                        wordPOSDict.update({canword:poscountlist})
                    
                    if Jyutping and not Jyutpingcountlist:
                        Jyutpingcountlist=[]
                        p_c_item={}
                        p_c_item.update({'Jyutping':Jyutping})
                        p_c_item.update({'count':1})
                        p_c_item.update({'第一次出现的句子':sentence})
                        Jyutpingcountlist.append(p_c_item)
                        
                        wordJyutpingDict.update({canword:Jyutpingcountlist})
                        if len(canword)>1:
                            twowordJyutpingDict.update({canword:Jyutpingcountlist})
                        
                    elif Jyutping and Jyutpingcountlist:
                        
                        pcLlen=len(Jyutpingcountlist)
                        
                        k=-1
                        tmpcount=0
                        for i in range(pcLlen):
                            tmpJyutping=Jyutpingcountlist[i].get('Jyutping','')
                            
                            if tmpJyutping==Jyutping:#这个canword的Jyutping词性已经出现过，count要加1
                                tmpcount=Jyutpingcountlist[i].get('count',0)+1
                                Jyutpingcountlist[i].update({'count':tmpcount})
                                
                                k=i
                                break
                       
                        if k==-1:#这个canword的Jyutping词性是第一次出现
                            p_c_item={}
                            
                            p_c_item.update({'Jyutping':Jyutping})
                            p_c_item.update({'count':1})
                            p_c_item.update({'第一次出现的句子':sentence})
                            Jyutpingcountlist.append(p_c_item)
                        
                        wordJyutpingDict.update({canword:Jyutpingcountlist})
                        if len(canword)>1:
                            twowordJyutpingDict.update({canword:Jyutpingcountlist})
                        
                    if not pos:
                        pos=' '
                    if canword not in punc:
                        mor=_MAP[pos]+'|'+Jyutping
                    else:
                        mor=canword
                    if judge_pure_english(canword):
                    
                        mor=_MAP[pos]+'|'+canword.lower()+'0'
                    morlist.append(mor)
                    mic=mic+" "+canword
                    mors=mors+" "+mor
                    j=j+3
                pre=tmp
                
                if len(sentence)!=len(tmpsent):
                    
                    
                    if not check(sentence):
                        with open(path4+'\\'+"Error.txt","a+",encoding="utf-8") as tf:
                            tf.write(name+":"+str(lineitems)+'\n')
                            tf.write(Film+":"+mic+'\n')
                            tf.write(Film+":"+mors+'\n')
                    else:
                        with open(path4+'\\'+"Warning.txt","a+",encoding="utf-8") as tf:
                            tf.write(name+":"+str(lineitems)+'\n')
                            tf.write(Film+":"+mic+'\n')
                            tf.write(Film+":"+mors+'\n')
                    if not os.path.exists(path6+'\\'+Film+".chat"):
                        wf_chat=open(path6+'\\'+Film+".chat","w",encoding="utf-8")
                        wf_chat.close()
                        
                    with open(path6+'\\'+Film+".chat","a+",encoding="utf-8") as tf:
                        
                        tf.write(str(sound_time)+'\n')
                        tf.write(mic+'\n')
                        tf.write(mors+'\n')
                else:
                    if not os.path.exists(path6+'\\'+Film+".chat"):
                        wf_chat=open(path6+'\\'+Film+".chat","w",encoding="utf-8")
                        wf_chat.close()
                        
                    with open(path6+'\\'+Film+".chat","a+",encoding="utf-8") as tf:
                        tf.write(str(sound_time)+'\n')
                        tf.write(mic+'\n')
                        tf.write(mors+'\n')
                    
                    
            lineNo=lineNo+1

twPOSDict={}
twJyutpingDict={}
twtJyutpingDict={}
for key in wordPOSDict:
    if len(wordPOSDict[key])==1:
        twPOSDict.update({key:wordPOSDict[key][0]['POS']})
with open(path4+'\\'+'hkccOnePOSdict.json',"w",encoding="utf-8") as wf:
    json.dump(twPOSDict,wf, indent=4, ensure_ascii=False)    
with open(path4+'\\'+'hkccPOSdict.json',"w",encoding="utf-8") as wf:
    json.dump(wordPOSDict,wf, indent=4, ensure_ascii=False)

for key in wordJyutpingDict:
    if len(wordJyutpingDict[key])==1:
        twJyutpingDict.update({key:wordJyutpingDict[key][0]['Jyutping']})
for key in twowordJyutpingDict:
    if len(twowordJyutpingDict[key])>1:
        twtJyutpingDict.update({key:twowordJyutpingDict[key]})

with open(path4+'\\'+'hkccOneJyutpingdict.json',"w",encoding="utf-8") as wf:
    json.dump(twJyutpingDict,wf, indent=4, ensure_ascii=False)

with open(path4+'\\'+'hkccJyutpingdict.json',"w",encoding="utf-8") as wf:
    json.dump(wordJyutpingDict,wf, indent=4, ensure_ascii=False)
with open(path4+'\\'+'twtJyutpingDict.json',"w",encoding="utf-8") as wf:
    json.dump(twtJyutpingDict,wf, indent=4, ensure_ascii=False)


    
        
